'''
Created on Jan 31, 2015

@author: eyaomai
'''
import sys
sys.path.append(sys.argv[0][:sys.argv[0].rfind('com/stocklens')])

from com.stocklens.stock.common.utils import Logging,  PIDUtils 
from com.stocklens.stock.data.crawlercommon import CrawlerConstants, CrawlerManager, CrawlerBase
from com.stocklens.stock.data.sina.sinacommon import SinaConstants
from bs4 import BeautifulSoup

class SinaStockRights(CrawlerManager):
    '''
    classdocs
    '''
    LOGGER_NAME_CRAWL_SSR = 'ssr'

    def __init__(self, json_config_file):
        '''
        Constructor
        '''
        super(SinaStockRights, self).__init__(json_config_file, 0.1, SinaRightsInfoCrawler.FAILUREURL_PATTERN)
        self.logger = Logging.getLogger(SinaStockRights.LOGGER_NAME_CRAWL_SSR)

    def start(self):
        super(SinaStockRights, self).start()

    def _generateTask(self, task, checkTaskList=True):
        if super(SinaStockRights, self)._generateTask(task, checkTaskList) is False:
            return False
         
        task[CrawlerConstants.PARA_CLASS] = SinaRightsInfoCrawler
        taskItem = self._taskList.pop(0)
        if len(taskItem)==6:
            stockid = taskItem
        else:
            rindex = taskItem.rfind('/')
            stockid = taskItem[rindex+1:rindex+7]
        task[SinaRightsInfoCrawler.PARA_STOCKID] = stockid    
        return True

    def _getNonCommonConfig(self, config):
        ignoreStop = config['ignoreStop']
        if ignoreStop.strip().lower()=='true':
            self.__ignoreStop = True
        else:
            self.__ignoreStop = False              

    def _initTask(self):
        if self.__ignoreStop:
            sql = 'SELECT distinct(stockid) FROM %s WHERE stockid NOT IN (SELECT stockid FROM %s)' % (SinaConstants.TABLE_SINA_STOCKINFO, SinaConstants.TABLE_SINA_STOCKINFO_STOP)
        else:
            sql = 'SELECT distinct(stockid) FROM %s' % (SinaConstants.TABLE_SINA_STOCKINFO)
        if self.dbProxy.execute(sql)>0:
            self._taskList.extend( [x[0] for x in self.dbProxy.cur.fetchall()])
        
class SinaRightsInfoCrawler(CrawlerBase):
    FAILUREURL_PATTERN = 'vISSUE_ShareBonus'
    URL = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vISSUE_ShareBonus/stockid/%s.phtml'
    PARA_STOCKID = 'stockid'
    NULL_CHARCATER = '--'
    SQL_NULL = 'null'
    def __init__(self, controller, dbProxy, request):
        super(SinaRightsInfoCrawler, self).__init__(controller, dbProxy, request)
        self.__stockid = request[SinaRightsInfoCrawler.PARA_STOCKID]
        if self.__stockid.startswith('6'):
            self.__market = 'SH'
        else:
            self.__market = 'SZ'
        self.__url = SinaRightsInfoCrawler.URL % self.__stockid 
        self.logger = Logging.getLogger(SinaStockRights.LOGGER_NAME_CRAWL_SSR)
    def run(self):
        super(SinaRightsInfoCrawler, self).run()
        
        self.logger.info('Start Crawl %s', self.__url)
        content = self._fetchContent(self.__url)
        status = CrawlerConstants.VAL_STATUS_FINISH
        if content is None:
            status = CrawlerConstants.VAL_STATUS_FAILURE
        else:
            try:
                content = content.replace('<th ', '<td ').replace('<th>','<td>')
                content = content.replace('//th>','//td>')
                self.__parseBonus(content, self.__url)
                self.__parseShare(content, self.__url)
                self._recoverFailure(self.__url)
            except Exception as e:
                self.logger.warn('Fail to parse %s:%s', self.__url, e)
                self._recordFailure(self.__url, 'Fail to parse')
                status = CrawlerConstants.VAL_STATUS_FAILURE
        self._reportDone(status)
        self.logger.info('Finish Crawl')
        
    
    def __parseRaw(self, content, url, ptype):
        lindex = content.find('<table id="%s"'%ptype)
        if lindex<0:
            self.logger.warn('No %s table for %s', ptype, url)
            return None
        soup = BeautifulSoup(content[lindex:])
        bonusTable = soup.findAll('table', {'id':ptype})
        if len(bonusTable)!=1:
            self.logger.warn('No %s table for %s', ptype, url)
            return None
        tbodys = bonusTable[0].findAll('tbody')
        if len(tbodys)!=1:
            self.logger.warn('%s len(tbody) is %d for %s ', ptype, len(tbodys), url)
            return None
        trs = tbodys[0].findAll('tr')
        if len(trs)==0:
            self.logger.warn('No trs under tbody for %s', ptype, url)
            return None
        if len(trs)==1:
            if len(trs[0].findAll('td'))==1:
                self.logger.info('No %s for %s', ptype, url)
                return None
        return trs
    
    def __parseBonus(self, content, url):
        trs = self.__parseRaw(content, url, 'sharebonus_1')
        if trs is None:
            return False
        values = list()
        for tr in trs:
            tds = tr.findAll('td')
            if len(tds)<9:
                continue
            publicDate = self.__transferValue(tds[0],True) 
            sendVol = self.__transferValue(tds[1])
            intoVol = self.__transferValue(tds[2])
            dividendAmount = self.__transferValue(tds[3])
            status = self.__transferValue(tds[4],True)
            rightDate = self.__transferValue(tds[5],True)
            registerDate = self.__transferValue(tds[6],True)
            listingDate=self.__transferValue(tds[7],True)
            value = '("%s","%s", %s, %s, %s, %s, %s,%s, %s,%s)' %\
                          (
                           self.__stockid,
                           self.__market,
                           publicDate,
                           sendVol,
                           intoVol,
                           dividendAmount,
                           rightDate,
                           registerDate,
                           listingDate,
                           status
                           )
            values.append(value)
        if len(values)>0:
            sql = 'DELETE FROM %s WHERE stockid="%s" AND market="%s"' %  (SinaConstants.TABLE_SINA_STOCK_BONUS, self.__stockid, self.__market)
            self._writeTable(sql)
            
            sql = 'INSERT INTO %s (stockid, market, publicdate, sendvol, intovol, dividendamount, rightdate, registerdate, listingdate, status) VALUES ' % (SinaConstants.TABLE_SINA_STOCK_BONUS)
            sql += ','.join(values)
            if self._writeTable(sql.encode('utf-8')) >0:
                self.totalNum+=len(values)
                
    def __parseShare(self, content, url):
        trs = self.__parseRaw(content, url, 'sharebonus_2')
        if trs is None:
            return False
        values = list()
        for tr in trs:
            tds = tr.findAll('td')
            if len(tds)<11:
                continue
            publicDate = self.__transferValue(tds[0], True) 
            allotvol = self.__transferValue(tds[1])
            alloprice = self.__transferValue(tds[2])
            basevol = self.__transferValue(tds[3])
            rightDate = self.__transferValue(tds[4], True)
            registerDate = self.__transferValue(tds[5], True)
            paymentstartdate = self.__transferValue(tds[6], True)
            paymentenddate = self.__transferValue(tds[7], True)
            listingDate=self.__transferValue(tds[8], True)
            totalamount = self.__transferValue(tds[9])
            values.append(
                          '("%s","%s", %s, %s, %s, %s, %s, %s, %s,%s, %s, %s)' %\
                          (
                           self.__stockid,
                           self.__market,
                           publicDate,
                           allotvol,
                           alloprice,
                           basevol,
                           rightDate,
                           registerDate,
                           paymentstartdate,
                           paymentenddate,
                           listingDate,
                           totalamount
                           )
                        )
        
        if len(values)>0:
            sql = 'DELETE FROM %s WHERE stockid="%s" AND market="%s"' %  (SinaConstants.TABLE_SINA_STOCK_ALLOTMENT, self.__stockid, self.__market)
            self._writeTable(sql)
            
            sql = 'INSERT INTO %s (stockid, market, publicdate, allotvol, allotprice, basevol, rightdate, paymentstartdate, paymentenddate, registerdate, listingdate, totalamount) VALUES ' % (SinaConstants.TABLE_SINA_STOCK_ALLOTMENT) 
            
            sql += ','.join(values)
            if self._writeTable(sql) >0:
                self.totalNum+=len(values)
    
    def __transferValue(self, soapElement, isStr=False):
        svalue = soapElement.text.strip()
        if svalue=='' or svalue== SinaRightsInfoCrawler.NULL_CHARCATER:
            return SinaRightsInfoCrawler.SQL_NULL
        if isStr:
            return '"'+svalue+'"'
        else:
            return svalue
    
if __name__ == '__main__':
    if PIDUtils.isPidFileExist('ssr'):
        print 'Previous SinaRights Crawler process is on-going, please stop it firstly'
        sys.exit(1)
    import os
    pid = os.getpid()
    PIDUtils.writePid('ssr', pid)
    Logging.initLogger('conf/crawler/crawler.logging.cfg')
    ssr = SinaStockRights('conf/crawler/ssr.cfg')
    ssr.start()
    pidutils = PIDUtils('ssr', ssr.shutDown, 5, ssr.logger)
    pidutils.start()
    sys.exit(0)
    